{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7d376606",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import awkward as ak\n",
    "import struct\n",
    "#data = np.memmap(\"users.avro\", np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "65123e0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "50216683",
   "metadata": {},
   "outputs": [],
   "source": [
    "from avro.datafile import DataFileWriter, DataFileReader\n",
    "from avro.io import DatumWriter, DatumReader\n",
    "import avro\n",
    "# {\n",
    "#        'name': 'avro.example.User',\n",
    "#    'type': 'record',\n",
    "#    'fields': [\n",
    "schema = {'name': 'nums', #'type': {\n",
    "                \"type\": \"array\",  \n",
    "                \"items\": {'name': 'nums1', #'type': {\n",
    "                \"type\": \"array\",  \n",
    "                \"items\": 'string'}}#}#]\n",
    "#}\n",
    "schema_parsed = avro.schema.Parse(json.dumps(schema))\n",
    "with open('users1.avro', 'wb') as f:\n",
    "    writer = DataFileWriter(f, DatumWriter(), schema_parsed)\n",
    "    writer.append([[\"fefef\"],[\"egege\"]])\n",
    "    writer.append([[\"Agee\",\"aegeg\"],[\"gr4gw\"]])\n",
    "    writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e7f5e413",
   "metadata": {},
   "outputs": [],
   "source": [
    "from avro.datafile import DataFileWriter, DataFileReader\n",
    "from avro.io import DatumWriter, DatumReader\n",
    "import avro\n",
    "schema = {\n",
    "    'name': 'avro.example.User',\n",
    "    'type': 'record',\n",
    "    'fields': [\n",
    "        {'name': 'name', 'type': 'string'},\n",
    "        {'name': 'ex4', 'type': 'bytes'},\n",
    "        {\"name\": \"ex3\", \"type\": {\"type\": \"fixed\", \"size\": 13, \"name\": \"ex3\"}},\n",
    "        {'name': 'hair', 'type': 'boolean', 'default': True},\n",
    "        {'name': 'ex', 'type': 'float'},\n",
    "        {'name': 'ex2', 'type': 'double'},\n",
    "        {'name': 'age', 'type': {'type': 'int'}},\n",
    "        {\"name\": \"Numbers\", \"type\": { \"type\" : \"enum\",  \"name\" : \"Numbers\", \"namespace\": \"data\", \"symbols\" : [ \"ONE\", \"TWO\", \"THREE\", \"FOUR\" ]}},\n",
    "        {'name': 'nums', 'type': {\n",
    "                \"type\": \"array\",  \n",
    "                \"items\": 'long'}},\n",
    "        { \"name\" : \"experience\", \"type\": [\"int\", \"null\"] },\n",
    "        \n",
    "    ]\n",
    "}\n",
    "schema_parsed = avro.schema.Parse(json.dumps(schema))\n",
    "with open('users.avro', 'wb') as f:\n",
    "    writer = DataFileWriter(f, DatumWriter(), schema_parsed)\n",
    "    writer.append({'name': 'Pierre-Simon Laplace','ex4': bytes(\"hello\",'utf-8'),'ex3': b\"like this one\",'hair': False,'age': 77, 'ex': 1.3435,'ex2': 4.4235,'Numbers': 'TWO', 'nums': [43,12], 'experience': 88})\n",
    "    writer.append({'name': 'John von Neumann','ex4': bytes(\"help\",'utf-8'),'ex3': b\"like this one\",'hair': True,'age': 53, 'ex': 5.123142134, 'ex2': 1.34535,'Numbers': 'THREE', 'nums': [57,23,4522]})\n",
    "    writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8743a231",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cd99cb11",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('users.avro', 'rb') as f:\n",
    "    reader = DataFileReader(f, DatumReader())\n",
    "    metadata = copy.deepcopy(reader.meta)\n",
    "    schema_from_file = json.loads(metadata['avro.schema'])\n",
    "    users = [user for user in reader]\n",
    "    reader.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca547d27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Schema that we specified:\n",
      " {'name': 'avro.example.User', 'type': 'record', 'fields': [{'name': 'name', 'type': 'string'}, {'name': 'ex4', 'type': 'bytes'}, {'name': 'ex3', 'type': {'type': 'fixed', 'size': 13, 'name': 'ex3'}}, {'name': 'hair', 'type': 'boolean', 'default': True}, {'name': 'ex', 'type': 'float'}, {'name': 'ex2', 'type': 'double'}, {'name': 'age', 'type': {'type': 'int'}}, {'name': 'Numbers', 'type': {'type': 'enum', 'name': 'Numbers', 'namespace': 'data', 'symbols': ['ONE', 'TWO', 'THREE', 'FOUR']}}, {'name': 'nums', 'type': {'type': 'array', 'items': 'long'}}, {'name': 'experience', 'type': ['int', 'null']}]}\n",
      "Schema that we parsed:\n",
      " {\"type\": \"record\", \"name\": \"User\", \"namespace\": \"avro.example\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"bytes\", \"name\": \"ex4\"}, {\"type\": {\"type\": \"fixed\", \"name\": \"ex3\", \"namespace\": \"avro.example\", \"size\": 13}, \"name\": \"ex3\"}, {\"type\": \"boolean\", \"name\": \"hair\", \"default\": true}, {\"type\": \"float\", \"name\": \"ex\"}, {\"type\": \"double\", \"name\": \"ex2\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": {\"type\": \"enum\", \"name\": \"Numbers\", \"namespace\": \"data\", \"symbols\": [\"ONE\", \"TWO\", \"THREE\", \"FOUR\"]}, \"name\": \"Numbers\"}, {\"type\": {\"type\": \"array\", \"items\": \"long\"}, \"name\": \"nums\"}, {\"type\": [\"int\", \"null\"], \"name\": \"experience\"}]}\n",
      "Schema from users.avro file:\n",
      " {'type': 'record', 'name': 'User', 'namespace': 'avro.example', 'fields': [{'type': 'string', 'name': 'name'}, {'type': 'bytes', 'name': 'ex4'}, {'type': {'type': 'fixed', 'name': 'ex3', 'namespace': 'avro.example', 'size': 13}, 'name': 'ex3'}, {'type': 'boolean', 'name': 'hair', 'default': True}, {'type': 'float', 'name': 'ex'}, {'type': 'double', 'name': 'ex2'}, {'type': 'int', 'name': 'age'}, {'type': {'type': 'enum', 'name': 'Numbers', 'namespace': 'data', 'symbols': ['ONE', 'TWO', 'THREE', 'FOUR']}, 'name': 'Numbers'}, {'type': {'type': 'array', 'items': 'long'}, 'name': 'nums'}, {'type': ['int', 'null'], 'name': 'experience'}]}\n",
      "Users:\n",
      " [{'name': 'Pierre-Simon Laplace', 'ex4': b'hello', 'ex3': b'like this one', 'hair': False, 'ex': 1.343500018119812, 'ex2': 4.4235, 'age': 77, 'Numbers': 'TWO', 'nums': [43, 12], 'experience': 88}, {'name': 'John von Neumann', 'ex4': b'help', 'ex3': b'like this one', 'hair': True, 'ex': 5.123142242431641, 'ex2': 1.34535, 'age': 53, 'Numbers': 'THREE', 'nums': [57, 23, 4522], 'experience': None}]\n"
     ]
    }
   ],
   "source": [
    "print(f'Schema that we specified:\\n {schema}')\n",
    "print(f'Schema that we parsed:\\n {schema_parsed}')\n",
    "print(f'Schema from users.avro file:\\n {schema_from_file}')\n",
    "print(f'Users:\\n {users}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dad9d6bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.memmap(\"users.avro\", np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e34300f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "832"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "22ec5fa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "ex = data[631:len(data)]\n",
    "val = ''\n",
    "for i in range(len(ex)):\n",
    "    if ex[i] < 10:\n",
    "        val = val+str(ex[i])\n",
    "    else:\n",
    "        val = val + chr(ex[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0bff7484",
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_varint(pos, data):\n",
    "     shift = 0\n",
    "     result = 0\n",
    "     while True:\n",
    "         i = data[pos]\n",
    "         pos += 1\n",
    "         result |= (i & 0x7f) << shift\n",
    "         shift += 7\n",
    "         if not (i & 0x80):\n",
    "             break\n",
    "     return pos, result\n",
    "\n",
    "def decode_zigzag(n):\n",
    "     return (n >> 1) ^ (-(n & 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "34af1a27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Obj\\x01\\x04\\x14avro.codec\\x08null\\x16avro.schema\\x82\\n{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"avro.example\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"bytes\", \"name\": \"ex4\"}, {\"type\": {\"type\": \"fixed\", \"name\": \"ex3\", \"namespace\": \"avro.example\", \"size\": 13}, \"name\": \"ex3\"}, {\"type\": \"boolean\", \"name\": \"hair\", \"default\": true}, {\"type\": \"float\", \"name\": \"ex\"}, {\"type\": \"double\", \"name\": \"ex2\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": {\"type\": \"enum\", \"name\": \"Numbers\", \"namespace\": \"data\", \"symbols\": [\"ONE\", \"TWO\", \"THREE\", \"FOUR\"]}, \"name\": \"Numbers\"}, {\"type\": {\"type\": \"array\", \"items\": \"long\"}, \"name\": \"nums\"}, {\"type\": [\"int\", \"null\"], \"name\": \"experience\"}]}\\x00\\xaf\\xeeNgz\\xe8\\xdf*\\xf4R\\xab\\xc2\\xd3\\x8eN\\x9d\\x04\\xf0\\x01(Pierre-Simon Laplace\\nhellolike this one\\x00\\xcf\\xf7\\xab?\\x8bl\\xe7\\xfb\\xa9\\xb1\\x11@\\x9a\\x01\\x02\\x04V\\x18\\x00\\x00\\xb0\\x01 John von Neumann\\x08helplike this one\\x01\\xc8\\xf0\\xa3@\\x11\\xc7\\xba\\xb8\\x8d\\x86\\xf5?j\\x04\\x06r.\\xd4F\\x00\\x02\\xaf\\xeeNgz\\xe8\\xdf*\\xf4R\\xab\\xc2\\xd3\\x8eN\\x9d'\n"
     ]
    }
   ],
   "source": [
    "ex = data[67:85]\n",
    "val1 = []\n",
    "for i in range(len(ex)):\n",
    "    val1.append(chr(ex[i]))\n",
    "print(data[:].tobytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fff8fce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class read_avro:\n",
    "    def __init__(self,file_name):\n",
    "        self.file_name = file_name\n",
    "        self._data = np.memmap(file_name, np.uint8)\n",
    "        self.field = []\n",
    "        _data = self._data\n",
    "        if not self.check_valid():\n",
    "            raise\n",
    "        pos, self.pairs = self.decode_varint(4, self._data)\n",
    "        self.pairs = self.decode_zigzag(self.pairs)\n",
    "        pos = self.cont_spec(pos)\n",
    "        pos = self.decode_block(pos)\n",
    "        tempos = pos\n",
    "        ind = 2\n",
    "        _exec_code = [f\"data = np.memmap(\\\"{self.file_name}\\\", np.uint8)\\npos=0\\n\",'''def decode_varint(pos, data):\\n\\tshift = 0\\n\\tresult = 0\\n\\twhile True:\\n\\t\\ti = data[pos]\\n\\t\\tpos += 1\\n\\t\\tresult |= (i & 0x7f) << shift\\n\\t\\tshift += 7\\n\\t\\tif not (i & 0x80):\\n\\t\\t\\tbreak\\n\\treturn pos, result\\ndef decode_zigzag(n):\\n\\treturn (n >> 1) ^ (-(n & 1))\\n\\n''',f\"field = {str(self.field)}\\n\",f\"for i in range({len(self.field)}):\\n\",\"    fields = data[field[i][0]:field[i][1]]\\n\",\"    while pos != len(fields):\"]\n",
    "        aform = []\n",
    "        dec = [\"import struct\\nimport numpy \\n\", 'con = {']\n",
    "        self.form,self._exec_code,self.count,dec = self.rec_exp_json_code(self.schema,_exec_code,ind,aform,0,dec)\n",
    "        dec.append(\"}\")\n",
    "        dec.append(\"\\n\")\n",
    "        head = \"\".join(dec)\n",
    "        body = \"\".join(self._exec_code)\n",
    "        #print(\"\".join(head+body))\n",
    "        loc = {}\n",
    "        exec(\"\".join(head+body),globals(),loc)\n",
    "        self.form = json.loads(\"\".join(self.form)[:-2])\n",
    "        con = loc['con']\n",
    "        for key in con.keys():\n",
    "            con[key] = np.array(con[key])\n",
    "        self.outarr = ak.from_buffers(self.form,2,con)\n",
    "    def decode_block(self,pos):\n",
    "        self.blocks = []\n",
    "        count = 0\n",
    "        while True:\n",
    "            print(pos)\n",
    "            pos, info = self.decode_varint(pos+17, self._data)#The byte is random at this position\n",
    "            info = self.decode_zigzag(info)\n",
    "            self.blocks.append((count,int(info)))\n",
    "            print(pos)\n",
    "            pos, info = self.decode_varint(pos, self._data)\n",
    "            info = self.decode_zigzag(info)\n",
    "            #print(pos,info)\n",
    "            self.field.append([pos, pos+int(info)])\n",
    "            pos = pos+info\n",
    "            if pos+16 == len(self._data):\n",
    "                break\n",
    "        return pos\n",
    "    def cont_spec(self, pos):\n",
    "        aa = -1\n",
    "        while aa != 2:\n",
    "            pos, dat = self.decode_varint(pos, self._data)\n",
    "            dat = self.decode_zigzag(dat)\n",
    "            val = self.ret_str(pos, pos+int(dat))\n",
    "            pos = pos+int(dat)\n",
    "            aa = self.check_special(val)\n",
    "            if aa == 0:\n",
    "                pos, dat = self.decode_varint(pos, self._data)\n",
    "                dat = self.decode_zigzag(dat)\n",
    "                val = self.ret_str(pos, pos+int(dat))\n",
    "                self.codec = val\n",
    "                pos = pos+int(dat)\n",
    "            elif aa == 1:\n",
    "\n",
    "                pos, dat = self.decode_varint(pos, self._data)\n",
    "                dat = self.decode_zigzag(dat)\n",
    "                val = self.ret_str(pos, pos+int(dat))\n",
    "                self.schema = json.loads(val)\n",
    "                pos = pos+int(dat)\n",
    "                return pos\n",
    "    \n",
    "    def check_special(self,val):\n",
    "        if val == \"avro.codec\":\n",
    "            return 0\n",
    "        elif val == \"avro.schema\":\n",
    "            return 1\n",
    "        else:\n",
    "            return 2\n",
    "            \n",
    "    def check_valid(self):\n",
    "        init = self.ret_str(0,4)\n",
    "        print(init)\n",
    "        if init == \"Obj\\x01\":\n",
    "            return True\n",
    "        else:\n",
    "            return False\n",
    "    \n",
    "    def ret_str(self, start, stop):\n",
    "        return self._data[start:stop].tobytes().decode(errors=\"surrogateescape\") \n",
    "    \n",
    "    def decode_varint(self, pos, _data):\n",
    "        shift = 0\n",
    "        result = 0\n",
    "        while True:\n",
    "            i = self._data[pos]\n",
    "            pos += 1\n",
    "            result |= (i & 0x7f) << shift\n",
    "            shift += 7\n",
    "            if not (i & 0x80):\n",
    "                break\n",
    "        return pos, result\n",
    "\n",
    "    def decode_zigzag(self, n):\n",
    "        return (n >> 1) ^ (-(n & 1))\n",
    "    \n",
    "    def dum_dat(self, dtype):\n",
    "        if dtype['type'] == 'int':\n",
    "            return \"np.int32(0)\"\n",
    "        if dtype['type'] == 'long':\n",
    "            return \"np.int64(0)\"\n",
    "        if dtype['type'] == 'float':\n",
    "            return \"np.float32(0)\"\n",
    "        if dtype['type'] == 'double':\n",
    "            return \"np.float64(0)\"\n",
    "        if dtype['type'] == 'boolean':\n",
    "            return \"0\"\n",
    "        if dtype['type'] == 'bytes':\n",
    "            return \"b'a'\"\n",
    "    \n",
    "    def rec_exp_json_code(self,file,_exec_code,ind,aform,count,dec):\n",
    "        if file[\"type\"] == \"null\":\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"record\":\n",
    "            #print(file[\"type\"],count)\n",
    "            temp = count\n",
    "            aform.append( '{\"class\" : \"RecordArray\",')\n",
    "            aform.append('\"contents\": {\\n')\n",
    "            for elem in file[\"fields\"]:\n",
    "                aform.append('\"'+elem[\"name\"]+'\"'+\": \")\n",
    "                aform,_exec_code,count,dec = self.rec_exp_json_code(elem,_exec_code,ind,aform,count+1,dec)\n",
    "            aform.append( f'}},\"form_key\": \"node{temp}\"}},\\n')\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"string\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": {{\"class\": \"NumpyArray\",\"primitive\": \"uint8\",\"parameters\": {{\"__array__\": \"char\"}},\"form_key\": \"node{count+1}\"}},\"parameters\": {{\"__array__\": \"string\"}},\"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-offsets'\"\n",
    "            var2 = f\" 'part0-node{count+1}-data'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [0],\")\n",
    "            dec.append(var2)\n",
    "            dec.append(\": [],\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-offsets'].append(np.uint8(out+con['part0-node{count}-offsets'][-1]))\")\n",
    "            #print(pos,abs(out))\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count+1}-data'].append(fields[pos:pos+abs(out)].tobytes().decode(errors=\\\"surrogateescape\\\"))\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(fields[pos:pos+abs(out)].tobytes().decode(errors=\\\"surrogateescape\\\") )\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos = pos + abs(out)\")\n",
    "            return aform,_exec_code,count+1,dec\n",
    "\n",
    "        elif file[\"type\"] == \"int\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\" : \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-data'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-data'].append(np.int32(out))\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(out)\")\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"long\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\" : \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-data'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(out)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-data'].append(np.int64(out))\")\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"float\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\" : \"NumpyArray\", \"primitive\": \"float32\", \"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-data'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(struct.Struct(\\\"<f\\\").unpack(fields[pos:pos+4].tobytes())[0])\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-data'].append(np.float32(struct.Struct(\\\"<f\\\").unpack(fields[pos:pos+4].tobytes())[0]))\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos = pos+4\")\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"double\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\" : \"NumpyArray\", \"primitive\": \"float64\", \"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-data'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(struct.Struct(\\\"<d\\\").unpack(fields[pos:pos+8].tobytes())[0])\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-data'].append(np.float64(struct.Struct(\\\"<d\\\").unpack(fields[pos:pos+8].tobytes())[0]))\")        \n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos = pos+8\")\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"boolean\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\" : \"NumpyArray\", \"primitive\": \"bool\", \"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-data'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(fields[pos])\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-data'].append(fields[pos])\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos = pos+1\")\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"bytes\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            astring = f'{{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": {{\"class\": \"NumpyArray\", \"primitive\": \"uint8\",\"parameters\": {{\"__array__\": \"char\"}},\"form_key\": \"node{count+1}\"}},\"parameters\": {{\"__array__\": \"string\"}},\"form_key\": \"node{count}\"}},\\n'\n",
    "            var1 = f\" 'part0-node{count+1}-data'\"\n",
    "            var2 = f\" 'part0-node{count}-offsets'\"\n",
    "            dec.append(var2)\n",
    "            dec.append(\": [0],\")\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            aform .append(astring)\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-offsets'].append(out+con['part0-node{count}-offsets'][-1])\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(fields[pos:pos+out].tobytes())\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count+1}-data'].append(fields[pos:pos+out].tobytes())\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos = pos+out\")\n",
    "            return aform,_exec_code,count+1,dec\n",
    "\n",
    "        elif isinstance(file[\"type\"],list):\n",
    "            #print(file[\"name\"])\n",
    "            temp = count\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = abs(decode_zigzag(inn))\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(\\\"index :\\\",out)\")\n",
    "            out = len(file[\"type\"])\n",
    "            if out == 2:\n",
    "                if \"null\" in file[\"type\"]:\n",
    "                    aform.append('{\"class\": \"ByteMaskedArray\",\"mask\": \"i8\",\"content\":\\n')\n",
    "                    var1 = f\" 'part0-node{count}-mask'\"\n",
    "                    dec.append(var1)\n",
    "                    dec.append(\": [],\")\n",
    "            temp = count\n",
    "            idxx = file['type'].index('null')\n",
    "            for i in range(out):\n",
    "                if file[\"type\"][i] == \"null\":\n",
    "                    _exec_code.append(\"\\n\"+\"    \"*(ind)+\"if out == {}:\".format(i))\n",
    "                    _exec_code.append(\"\\n\"+\"    \"*(ind+1)+f\"con['part0-node{temp}-mask'].append(np.int8(False))\".format())\n",
    "                    _exec_code.append(\"\\n\"+\"    \"*(ind+1)+f\"con['part0-node{temp+1}-data'].append({self.dum_dat({'type': file['type'][1-idxx]})})\")\n",
    "                else:\n",
    "                    _exec_code.append(\"\\n\"+\"    \"*(ind)+\"if out == {}:\".format(i))\n",
    "                    _exec_code.append(\"\\n\"+\"    \"*(ind+1)+f\"con['part0-node{count}-mask'].append(np.int8(True))\")\n",
    "                    aform,_exec_code,count,dec = self.rec_exp_json_code({\"type\": file[\"type\"][i]},_exec_code,ind+1, aform,count+1,dec)\n",
    "            aform.append(f'\"valid_when\": true,\"form_key\": \"node{temp}\"}}\\n')\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif isinstance(file[\"type\"],dict):\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform,_exec_code,count,dec = self.rec_exp_json_code(file[\"type\"],_exec_code,ind, aform,count,dec)\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"fixed\":\n",
    "            #print(file[\"type\"],count)\n",
    "            var1 = \" 'part0-node{}-data'\".format(count+1)\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            aform.append(f'{{\"class\": \"RegularArray\",\"content\": {{\"class\": \"NumpyArray\",\"primitive\": \"uint8\",\"form_key\": \"node{count+1}\"}},\"size\": {file[\"size\"]},\"form_key\": \"node{count}\"}},\\n')\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"lenn = {}\".format(file[\"size\"]))\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(fields[pos:pos+lenn].tobytes())\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"for i in range(lenn):\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*(ind+1)+f\"con['part0-node{count+1}-data'].append(np.uint8(fields[i]))\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos = pos+lenn\")\n",
    "            return aform,_exec_code,count+1,dec\n",
    "\n",
    "        elif file[\"type\"] == \"enum\":\n",
    "            #print(file[\"name\"])\n",
    "            #print(file[\"type\"],count)\n",
    "            aform.append(f'{{\"class\": \"IndexedArray64\",\"index\": \"i64\",\"content\": {{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": {{\"class\": \"NumpyArray\",\"primitive\": \"uint8\",\"parameters\": {{\"__array__\": \"char\"}},\"form_key\": \"node{count+2}\"}},\"parameters\": {{\"__array__\": \"string\"}},\"form_key\": \"node{count+1}\"}}, \"parameters\": {{\"__array__\": \"categorical\"}}, \"form_key\": \"node{count}\"}},\\n')\n",
    "            var1 = f\" 'part0-node{count}-index'\"\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [],\")\n",
    "            tempar = file['symbols']\n",
    "            offset, dat = [0] , []\n",
    "            prev = 0\n",
    "            for i in range(len(tempar)):\n",
    "                offset.append(len(tempar[i])+prev)\n",
    "                prev = offset[-1]\n",
    "            for i in range(len(tempar)):\n",
    "                for elem in range(len(tempar[i])):\n",
    "                    dat.append(elem)\n",
    "            var2 = f\" 'part0-node{count+1}-offsets': {str(offset)},\"\n",
    "            dec.append(var2)\n",
    "            var2 = f\" 'part0-node{count+2}-data': {str(dat)},\"\n",
    "            dec.append(var2)\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-index'].append(np.int64(out))\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(out)\")\n",
    "            return aform,_exec_code,count+2,dec\n",
    "\n",
    "        elif file[\"type\"] == \"array\":\n",
    "            #print(file[\"name\"])\n",
    "            temp = count\n",
    "            var1 = \" 'part0-node{}-offsets'\".format(count)\n",
    "            dec.append(var1)\n",
    "            dec.append(\": [0],\")\n",
    "            aform.append('{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": ')\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"print(\\\"length\\\",out)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+f\"con['part0-node{count}-offsets'].append(out+con['part0-node{count}-offsets'][-1])\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"for i in range(out):\")\n",
    "            aform,_exec_code,count,dec = self.rec_exp_json_code({\"type\": file[\"items\"]},_exec_code,ind+1,aform,count+1,dec)\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "            _exec_code.append(\"\\n\"+\"    \"*ind+'''if out != 0:''')\n",
    "            _exec_code.append(\"\\n\"+\"    \"*(ind+1)+\"raise\")\n",
    "            aform.append( f'\"form_key\": \"node{temp}\"}},\\n')\n",
    "            return aform,_exec_code,count,dec\n",
    "\n",
    "        elif file[\"type\"] == \"map\":\n",
    "            #print(file[\"name\"])\n",
    "    #         _exec_code.append(\"\\npos, inn = decode_varint(pos,fields)\"\n",
    "    #         _exec_code.append(\"\\nout = abs(decode_zigzag(inn))\"\n",
    "    #         _exec_code.append(\"\\nprint(\\\"length\\\",out)\"\n",
    "    #         _exec_code.append(\"\\nfor i in range(out):\"\n",
    "    #         _exec_code.append(\"\\n\"+\"    \"*(ind+1)+\"print(\\\"{{\\\")\"\n",
    "    #         _exec_code = _exec_code+aa\n",
    "    #         _exec_code = _exec_code+bb\n",
    "    #         _exec_code = _exec_code+ccfa\n",
    "    #         _exec_code = _exec_code+dd\n",
    "    #         _exec_code = _exec_code+ee\n",
    "    #         pos,_exec_code,count = self.rec_exp_json_code({\"type\": file[\"values\"]},fields,pos,_exec_code,ind+1, aform,count+1)\n",
    "    #         _exec_code.append(\"\\n    \"*(ind+1)+\"print(\\\":\\\")\"\n",
    "    #         _exec_code = _exec_code+ff\n",
    "    #         pos,_exec_code,count = self.rec_exp_json_code({\"type\": file[\"values\"]},fields,pos,_exec_code,ind+1, aform,count+1)\n",
    "    #         _exec_code.append(\"\\n\"+\"    \"*(ind+1)+\"print(\\\"}}\\\")\"\n",
    "    #         _exec_code.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\"\n",
    "    #         jj = \"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\"\n",
    "    #         kk = \"\\n\"+\"    \"*ind+'''if out != 0:\n",
    "    #     raise\n",
    "    #             '''\n",
    "    #         _exec_code = _exec_code+gg\n",
    "    #         _exec_code = _exec_code+hh\n",
    "    #         _exec_code = _exec_code+jj\n",
    "    #         _exec_code = _exec_code+kk\n",
    "            raise NotImplementedError\n",
    "    #        return aform,_exec_code,count\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e939d6cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obj\u0001\n",
      "99\n",
      "117\n",
      "length 2\n",
      "length 1\n",
      "fefef\n",
      "length 1\n",
      "egege\n",
      "length 2\n",
      "length 2\n",
      "Agee\n",
      "aegeg\n",
      "length 1\n",
      "gr4gw\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'class': 'ListOffsetArray64',\n",
       " 'offsets': 'i64',\n",
       " 'content': {'class': 'ListOffsetArray64',\n",
       "  'offsets': 'i64',\n",
       "  'content': {'class': 'ListOffsetArray64',\n",
       "   'offsets': 'i64',\n",
       "   'content': {'class': 'NumpyArray',\n",
       "    'primitive': 'uint8',\n",
       "    'parameters': {'__array__': 'char'},\n",
       "    'form_key': 'node3'},\n",
       "   'parameters': {'__array__': 'string'},\n",
       "   'form_key': 'node2'},\n",
       "  'form_key': 'node1'},\n",
       " 'form_key': 'node0'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa = read_avro(\"users1.avro\")\n",
    "aa.form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "32619c2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<ListOffsetArray64>\n",
       "    <offsets><Index64 i=\"[0 2 4]\" offset=\"0\" length=\"3\" at=\"0x0000019bebf0\"/></offsets>\n",
       "    <content><ListOffsetArray64>\n",
       "        <offsets><Index64 i=\"[0 1 2 4 5]\" offset=\"0\" length=\"5\" at=\"0x000002c913a0\"/></offsets>\n",
       "        <content><ListOffsetArray64>\n",
       "            <parameters>\n",
       "                <param key=\"__array__\">\"string\"</param>\n",
       "            </parameters>\n",
       "            <offsets><Index64 i=\"[0 5 10 14 19 24]\" offset=\"0\" length=\"6\" at=\"0x000002bf7a00\"/></offsets>\n",
       "            <content><NumpyArray format=\"B\" shape=\"100\" data=\"102 0 0 0 101 ... 0 119 0 0 0\" at=\"0x000002c91330\">\n",
       "                <parameters>\n",
       "                    <param key=\"__array__\">\"char\"</param>\n",
       "                </parameters>\n",
       "            </NumpyArray></content>\n",
       "        </ListOffsetArray64></content>\n",
       "    </ListOffsetArray64></content>\n",
       "</ListOffsetArray64>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa.outarr.layout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3da27a66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obj\u0001\n",
      "67\n",
      "85\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'type': 'array', 'items': 'int'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa = read_header(\"users1.avro\")\n",
    "#aa.schema\n",
    "#data = np.memmap(\"users1.avro\", np.uint8)\n",
    "#data\n",
    "aa.schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7dad7f9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dum_dat(dtype):\n",
    "    print(dtype)\n",
    "    if dtype['type'] == 'int':\n",
    "        return \"np.int32(0)\"\n",
    "    if dtype['type'] == 'long':\n",
    "        return \"np.int64(0)\"\n",
    "    if dtype['type'] == 'float':\n",
    "        return \"np.float32(0)\"\n",
    "    if dtype['type'] == 'double':\n",
    "        return \"np.float64(0)\"\n",
    "    if dtype['type'] == 'boolean':\n",
    "        return \"0\"\n",
    "    if dtype['type'] == 'bytes':\n",
    "        return \"b'a'\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e4b7ab66",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rec_exp_json_code(file,fields,pos,strr,ind,aform,count,dec):\n",
    "    if file[\"type\"] == \"null\":\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"record\":\n",
    "        #print(file[\"type\"],count)\n",
    "        temp = count\n",
    "        aform.append( '{\"class\" : \"RecordArray\",')\n",
    "        aform.append('\"contents\": {\\n')\n",
    "        for elem in file[\"fields\"]:\n",
    "            aform.append('\"'+elem[\"name\"]+'\"'+\": \")\n",
    "            aform,strr,count,dec = rec_exp_json_code(elem,fields,pos,strr,ind,aform,count+1,dec)\n",
    "        aform.append( '}},\"form_key\": \"node{}\"}},\\n'.format(temp))\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"string\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": {{\"class\": \"NumpyArray\",\"primitive\": \"uint8\",\"parameters\": {{\"__array__\": \"char\"}},\"form_key\": \"node{}\"}},\"parameters\": {{\"__array__\": \"string\"}},\"form_key\": \"node{}\"}},\\n'.format(count+1,count))\n",
    "        var1 = \" 'part0-node{}-offsets'\".format(count)\n",
    "        var2 = \" 'part0-node{}-data'\".format(count+1)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [0],\")\n",
    "        dec.append(var2)\n",
    "        dec.append(\": [],\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-offsets'].append(np.uint8(out+con['part0-node{}-offsets'][-1]))\".format(count,count))\n",
    "        #print(pos,abs(out))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(fields[pos:pos+abs(out)].tobytes().decode(errors=\\\"surrogateescape\\\"))\".format(count+1))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(fields[pos:pos+abs(out)].tobytes().decode(errors=\\\"surrogateescape\\\") )\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos = pos + abs(out)\")\n",
    "        return aform,strr,count+1,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"int\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\" : \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node{}\"}},\\n'.format(count))\n",
    "        var1 = \" 'part0-node{}-data'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(np.int32(out))\".format(count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(out)\")\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"long\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\" : \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node{}\"}},\\n'.format(count))\n",
    "        var1 = \" 'part0-node{}-data'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(out)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(np.int64(out))\".format(count))\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"float\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\" : \"NumpyArray\", \"primitive\": \"float32\", \"form_key\": \"node{}\"}},\\n'.format(count))\n",
    "        var1 = \" 'part0-node{}-data'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(struct.Struct(\\\"<f\\\").unpack(fields[pos:pos+4].tobytes())[0])\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(np.float32(struct.Struct(\\\"<f\\\").unpack(fields[pos:pos+4].tobytes())[0]))\".format(count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos = pos+4\")\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"double\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\" : \"NumpyArray\", \"primitive\": \"float64\", \"form_key\": \"node{}\"}},\\n'.format(count))\n",
    "        var1 = \" 'part0-node{}-data'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(struct.Struct(\\\"<d\\\").unpack(fields[pos:pos+8].tobytes())[0])\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(np.float64(struct.Struct(\\\"<d\\\").unpack(fields[pos:pos+8].tobytes())[0]))\".format(count))        \n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos = pos+8\")\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"boolean\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\" : \"NumpyArray\", \"primitive\": \"bool\", \"form_key\": \"node{}\"}},\\n'.format(count))\n",
    "        var1 = \" 'part0-node{}-data'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(fields[pos])\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(fields[pos])\".format(count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos = pos+1\")\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"bytes\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        astring = '{{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": {{\"class\": \"NumpyArray\", \"primitive\": \"uint8\",\"parameters\": {{\"__array__\": \"char\"}},\"form_key\": \"node{}\"}},\"parameters\": {{\"__array__\": \"string\"}},\"form_key\": \"node{}\"}},\\n'.format(count+1,count)\n",
    "        var1 = \" 'part0-node{}-data'\".format(count+1)\n",
    "        var2 = \" 'part0-node{}-offsets'\".format(count)\n",
    "        dec.append(var2)\n",
    "        dec.append(\": [0],\")\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        aform .append(astring)\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-offsets'].append(out+con['part0-node{}-offsets'][-1])\".format(count,count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(fields[pos:pos+out].tobytes())\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-data'].append(fields[pos:pos+out].tobytes())\".format(count+1))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos = pos+out\")\n",
    "        return aform,strr,count+1,dec\n",
    "    \n",
    "    elif isinstance(file[\"type\"],list):\n",
    "        #print(file[\"name\"])\n",
    "        temp = count\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = abs(decode_zigzag(inn))\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(\\\"index :\\\",out)\")\n",
    "        out = len(file[\"type\"])\n",
    "        if out == 2:\n",
    "            if \"null\" in file[\"type\"]:\n",
    "                aform.append('{\"class\": \"ByteMaskedArray\",\"mask\": \"i8\",\"content\":\\n')\n",
    "                var1 = \" 'part0-node{}-mask'\".format(count)\n",
    "                dec.append(var1)\n",
    "                dec.append(\": [],\")\n",
    "        temp = count\n",
    "        idxx = file['type'].index('null')\n",
    "        for i in range(out):\n",
    "            if file[\"type\"][i] == \"null\":\n",
    "                strr.append(\"\\n\"+\"    \"*(ind)+\"if out == {}:\".format(i))\n",
    "                strr.append(\"\\n\"+\"    \"*(ind+1)+\"con['part0-node{}-mask'].append(np.int8(False))\".format(temp))\n",
    "                strr.append(\"\\n\"+\"    \"*(ind+1)+\"con['part0-node{}-data'].append({})\".format(temp+1,dum_dat({'type': file['type'][1-idxx]})))\n",
    "            else:\n",
    "                strr.append(\"\\n\"+\"    \"*(ind)+\"if out == {}:\".format(i))\n",
    "                strr.append(\"\\n\"+\"    \"*(ind+1)+\"con['part0-node{}-mask'].append(np.int8(True))\".format(count))\n",
    "                aform,strr,count,dec = rec_exp_json_code({\"type\": file[\"type\"][i]},fields,pos,strr,ind+1, aform,count+1,dec)\n",
    "        aform.append('\"valid_when\": true,\"form_key\": \"node{}\"}}\\n'.format(temp))\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif isinstance(file[\"type\"],dict):\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform,strr,count,dec = rec_exp_json_code(file[\"type\"],fields,pos,strr,ind, aform,count,dec)\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"fixed\":\n",
    "        #print(file[\"type\"],count)\n",
    "        var1 = \" 'part0-node{}-data'\".format(count+1)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        aform.append('{{\"class\": \"RegularArray\",\"content\": {{\"class\": \"NumpyArray\",\"primitive\": \"uint8\",\"form_key\": \"node{}\"}},\"size\": {},\"form_key\": \"node{}\"}},\\n'.format(count+1,file[\"size\"],count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"lenn = {}\".format(file[\"size\"]))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(fields[pos:pos+lenn].tobytes())\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"for i in range(lenn):\")\n",
    "        strr.append(\"\\n\"+\"    \"*(ind+1)+\"con['part0-node{}-data'].append(np.uint8(fields[i]))\".format(count+1))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos = pos+lenn\")\n",
    "        return aform,strr,count+1,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"enum\":\n",
    "        #print(file[\"name\"])\n",
    "        #print(file[\"type\"],count)\n",
    "        aform.append('{{\"class\": \"IndexedArray64\",\"index\": \"i64\",\"content\": {{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": {{\"class\": \"NumpyArray\",\"primitive\": \"uint8\",\"parameters\": {{\"__array__\": \"char\"}},\"form_key\": \"node{}\"}},\"parameters\": {{\"__array__\": \"string\"}},\"form_key\": \"node{}\"}}, \"parameters\": {{\"__array__\": \"categorical\"}}, \"form_key\": \"node{}\"}},\\n'.format(count+2,count+1,count))\n",
    "        var1 = \" 'part0-node{}-index'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [],\")\n",
    "        tempar = file['symbols']\n",
    "        offset, dat = [0] , []\n",
    "        prev = 0\n",
    "        for i in range(len(tempar)):\n",
    "            offset.append(len(tempar[i])+prev)\n",
    "            prev = offset[-1]\n",
    "        for i in range(len(tempar)):\n",
    "            for elem in range(len(tempar[i])):\n",
    "                dat.append(elem)\n",
    "        var2 = \" 'part0-node{}-offsets': {},\".format(count+1,str(offset))\n",
    "        dec.append(var2)\n",
    "        var2 = \" 'part0-node{}-data': {},\".format(count+2,str(dat))\n",
    "        dec.append(var2)\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-index'].append(np.int64(out))\".format(count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(out)\")\n",
    "        return aform,strr,count+2,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"array\":\n",
    "        #print(file[\"name\"])\n",
    "        temp = count\n",
    "        var1 = \" 'part0-node{}-offsets'\".format(count)\n",
    "        dec.append(var1)\n",
    "        dec.append(\": [0],\")\n",
    "        aform.append('{\"class\": \"ListOffsetArray64\",\"offsets\": \"i64\",\"content\": ')\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"print(\\\"length\\\",out)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"con['part0-node{}-offsets'].append(out+con['part0-node{}-offsets'][-1])\".format(count,count))\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"for i in range(out):\")\n",
    "        aform,strr,count,dec = rec_exp_json_code({\"type\": file[\"items\"]},fields,pos,strr,ind+1,aform,count+1,dec)\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\")\n",
    "        strr.append(\"\\n\"+\"    \"*ind+'''if out != 0:''')\n",
    "        strr.append(\"\\n\"+\"    \"*(ind+1)+\"raise\")\n",
    "        aform.append( '\"form_key\": \"node{}\"}},\\n'.format(temp))\n",
    "        return aform,strr,count,dec\n",
    "    \n",
    "    elif file[\"type\"] == \"map\":\n",
    "        #print(file[\"name\"])\n",
    "#         strr.append(\"\\npos, inn = decode_varint(pos,fields)\"\n",
    "#         strr.append(\"\\nout = abs(decode_zigzag(inn))\"\n",
    "#         strr.append(\"\\nprint(\\\"length\\\",out)\"\n",
    "#         strr.append(\"\\nfor i in range(out):\"\n",
    "#         strr.append(\"\\n\"+\"    \"*(ind+1)+\"print(\\\"{{\\\")\"\n",
    "#         strr = strr+aa\n",
    "#         strr = strr+bb\n",
    "#         strr = strr+ccfa\n",
    "#         strr = strr+dd\n",
    "#         strr = strr+ee\n",
    "#         pos,strr,count = rec_exp_json_code({\"type\": file[\"values\"]},fields,pos,strr,ind+1, aform,count+1)\n",
    "#         strr.append(\"\\n    \"*(ind+1)+\"print(\\\":\\\")\"\n",
    "#         strr = strr+ff\n",
    "#         pos,strr,count = rec_exp_json_code({\"type\": file[\"values\"]},fields,pos,strr,ind+1, aform,count+1)\n",
    "#         strr.append(\"\\n\"+\"    \"*(ind+1)+\"print(\\\"}}\\\")\"\n",
    "#         strr.append(\"\\n\"+\"    \"*ind+\"pos, inn = decode_varint(pos,fields)\"\n",
    "#         jj = \"\\n\"+\"    \"*ind+\"out = decode_zigzag(inn)\"\n",
    "#         kk = \"\\n\"+\"    \"*ind+'''if out != 0:\n",
    "#     raise\n",
    "#             '''\n",
    "#         strr = strr+gg\n",
    "#         strr = strr+hh\n",
    "#         strr = strr+jj\n",
    "#         strr = strr+kk\n",
    "        raise NotImplementedError\n",
    "#        return aform,strr,count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3f80570a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'type': 'int'}\n",
      "import struct\n",
      "import numpy \n",
      "con = { 'part0-node1-offsets': [0], 'part0-node2-data': [], 'part0-node3-offsets': [0], 'part0-node4-data': [], 'part0-node6-data': [], 'part0-node7-data': [], 'part0-node8-data': [], 'part0-node9-data': [], 'part0-node10-data': [], 'part0-node11-index': [], 'part0-node12-offsets': [0, 3, 6, 11, 15], 'part0-node13-data': [0, 1, 2, 0, 1, 2, 0, 1, 2, 3, 4, 0, 1, 2, 3], 'part0-node14-offsets': [0], 'part0-node15-data': [], 'part0-node16-mask': [], 'part0-node17-data': [],}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ind = 1\n",
    "strr = [\"data = np.memmap(\\\"{}\\\", np.uint8)\\nfields = data[{}:]\\npos=0\\n\",'''def decode_varint(pos, data):\n",
    "     shift = 0\n",
    "     result = 0\n",
    "     while True:\n",
    "         i = data[pos]\n",
    "         pos += 1\n",
    "         result |= (i & 0x7f) << shift\n",
    "         shift += 7\n",
    "         if not (i & 0x80):\n",
    "             break\n",
    "     return pos, result\\n\n",
    "def decode_zigzag(n):\n",
    "     return (n >> 1) ^ (-(n & 1))\\n\\n''',\"while pos+16 != len(fields):\".format(aa.file_name,aa.field[0][0])]\n",
    "#for i in range(2):\n",
    "aform = []\n",
    "dec = [\"import struct\\nimport numpy \\n\", 'con = {']\n",
    "form,strr,count,dec = rec_exp_json_code(aa.schema,aa._data[aa.field[0][0]:aa.field[0][1]],pos,strr,ind,aform,0,dec)\n",
    "dec.append(\"}\")\n",
    "dec.append(\"\\n\")\n",
    "bb = \"\".join(dec)\n",
    "cc = \"\".join(strr)\n",
    "print(\"\".join(dec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "80438447",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pierre-Simon Laplace\n",
      "b'hello'\n",
      "b'like this one'\n",
      "0\n",
      "1.343500018119812\n",
      "4.4235\n",
      "77\n",
      "1\n",
      "length 2\n",
      "43\n",
      "12\n",
      "index : 0\n",
      "88\n",
      "John von Neumann\n",
      "b'help'\n",
      "b'like this one'\n",
      "1\n",
      "5.123142242431641\n",
      "1.34535\n",
      "53\n",
      "2\n",
      "length 3\n",
      "57\n",
      "23\n",
      "4522\n",
      "index : 1\n"
     ]
    }
   ],
   "source": [
    "exec(\"\".join(bb+cc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d38323e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = json.loads(\"\".join(form)[:-2])\n",
    "#print(\"\".join(pos))\n",
    "#print(json.dumps(pos, indent = 1))\n",
    "for key in con.keys():\n",
    "    con[key] = np.array(con[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "79fb50fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class': 'RecordArray',\n",
       " 'contents': {'name': {'class': 'ListOffsetArray64',\n",
       "   'offsets': 'i64',\n",
       "   'content': {'class': 'NumpyArray',\n",
       "    'primitive': 'uint8',\n",
       "    'parameters': {'__array__': 'char'},\n",
       "    'form_key': 'node2'},\n",
       "   'parameters': {'__array__': 'string'},\n",
       "   'form_key': 'node1'},\n",
       "  'ex4': {'class': 'ListOffsetArray64',\n",
       "   'offsets': 'i64',\n",
       "   'content': {'class': 'NumpyArray',\n",
       "    'primitive': 'uint8',\n",
       "    'parameters': {'__array__': 'char'},\n",
       "    'form_key': 'node4'},\n",
       "   'parameters': {'__array__': 'string'},\n",
       "   'form_key': 'node3'},\n",
       "  'ex3': {'class': 'RegularArray',\n",
       "   'content': {'class': 'NumpyArray',\n",
       "    'primitive': 'uint8',\n",
       "    'form_key': 'node6'},\n",
       "   'size': 13,\n",
       "   'form_key': 'node5'},\n",
       "  'hair': {'class': 'NumpyArray', 'primitive': 'bool', 'form_key': 'node7'},\n",
       "  'ex': {'class': 'NumpyArray', 'primitive': 'float32', 'form_key': 'node8'},\n",
       "  'ex2': {'class': 'NumpyArray', 'primitive': 'float64', 'form_key': 'node9'},\n",
       "  'age': {'class': 'NumpyArray', 'primitive': 'int32', 'form_key': 'node10'},\n",
       "  'Numbers': {'class': 'IndexedArray64',\n",
       "   'index': 'i64',\n",
       "   'content': {'class': 'ListOffsetArray64',\n",
       "    'offsets': 'i64',\n",
       "    'content': {'class': 'NumpyArray',\n",
       "     'primitive': 'uint8',\n",
       "     'parameters': {'__array__': 'char'},\n",
       "     'form_key': 'node13'},\n",
       "    'parameters': {'__array__': 'string'},\n",
       "    'form_key': 'node12'},\n",
       "   'parameters': {'__array__': 'categorical'},\n",
       "   'form_key': 'node11'},\n",
       "  'nums': {'class': 'ListOffsetArray64',\n",
       "   'offsets': 'i64',\n",
       "   'content': {'class': 'NumpyArray',\n",
       "    'primitive': 'int64',\n",
       "    'form_key': 'node15'},\n",
       "   'form_key': 'node14'},\n",
       "  'experience': {'class': 'ByteMaskedArray',\n",
       "   'mask': 'i8',\n",
       "   'content': {'class': 'NumpyArray',\n",
       "    'primitive': 'int32',\n",
       "    'form_key': 'node17'},\n",
       "   'valid_when': True,\n",
       "   'form_key': 'node16'}},\n",
       " 'form_key': 'node0'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "47a4f8e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Numbers': '\\x00\\x00\\x00',\n",
       "  'age': 77,\n",
       "  'ex': 1.343500018119812,\n",
       "  'ex2': 4.4235,\n",
       "  'ex3': [40, 80, 105, 101, 114, 114, 101, 45, 83, 105, 109, 111, 110],\n",
       "  'ex4': 'hello',\n",
       "  'experience': 88,\n",
       "  'hair': False,\n",
       "  'name': 'P\\x00\\x00\\x00i\\x00\\x00\\x00e\\x00\\x00\\x00r\\x00\\x00\\x00r\\x00\\x00\\x00',\n",
       "  'nums': [43, 12]},\n",
       " {'Numbers': '\\x00\\x00\\x01\\x00\\x00',\n",
       "  'age': 53,\n",
       "  'ex': 5.123142242431641,\n",
       "  'ex2': 1.34535,\n",
       "  'ex3': [40, 80, 105, 101, 114, 114, 101, 45, 83, 105, 109, 111, 110],\n",
       "  'ex4': 'help',\n",
       "  'experience': None,\n",
       "  'hair': True,\n",
       "  'name': 'e\\x00\\x00\\x00-\\x00\\x00\\x00S\\x00\\x00\\x00i\\x00\\x00\\x00',\n",
       "  'nums': [57, 23, 4522]}]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tempar = ak.from_buffers(pos,2,con)\n",
    "#tempar['ex4']\n",
    "tempar.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1ab07ff9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"class\": \"RecordArray\",\n",
      "    \"contents\": {\n",
      "        \"px\": {\n",
      "            \"class\": \"ListOffsetArray64\",\n",
      "            \"offsets\": \"i64\",\n",
      "            \"content\": {\n",
      "                \"class\": \"RecordArray\",\n",
      "                \"contents\": {\n",
      "                    \"key\": {\n",
      "                        \"class\": \"ListOffsetArray64\",\n",
      "                        \"offsets\": \"i64\",\n",
      "                        \"content\": {\n",
      "                            \"class\": \"NumpyArray\",\n",
      "                            \"itemsize\": 1,\n",
      "                            \"format\": \"B\",\n",
      "                            \"primitive\": \"uint8\",\n",
      "                            \"parameters\": {\n",
      "                                \"__array__\": \"char\"\n",
      "                            },\n",
      "                            \"form_key\": \"node4\"\n",
      "                        },\n",
      "                        \"parameters\": {\n",
      "                            \"__array__\": \"string\"\n",
      "                        },\n",
      "                        \"form_key\": \"node3\"\n",
      "                    },\n",
      "                    \"value\": {\n",
      "                        \"class\": \"NumpyArray\",\n",
      "                        \"itemsize\": 8,\n",
      "                        \"format\": \"l\",\n",
      "                        \"primitive\": \"int64\",\n",
      "                        \"form_key\": \"node5\"\n",
      "                    }\n",
      "                },\n",
      "                \"form_key\": \"node2\"\n",
      "            },\n",
      "            \"form_key\": \"node1\"\n",
      "        },\n",
      "        \"py\": {\n",
      "            \"class\": \"RecordArray\",\n",
      "            \"contents\": {\n",
      "                \"px\": {\n",
      "                    \"class\": \"NumpyArray\",\n",
      "                    \"itemsize\": 1,\n",
      "                    \"format\": \"?\",\n",
      "                    \"primitive\": \"bool\",\n",
      "                    \"form_key\": \"node7\"\n",
      "                },\n",
      "                \"py\": {\n",
      "                    \"class\": \"NumpyArray\",\n",
      "                    \"itemsize\": 8,\n",
      "                    \"format\": \"d\",\n",
      "                    \"primitive\": \"float64\",\n",
      "                    \"form_key\": \"node8\"\n",
      "                },\n",
      "                \"pz\": {\n",
      "                    \"class\": \"NumpyArray\",\n",
      "                    \"itemsize\": 8,\n",
      "                    \"format\": \"d\",\n",
      "                    \"primitive\": \"float64\",\n",
      "                    \"form_key\": \"node9\"\n",
      "                },\n",
      "                \"E\": {\n",
      "                    \"class\": \"NumpyArray\",\n",
      "                    \"itemsize\": 8,\n",
      "                    \"format\": \"d\",\n",
      "                    \"primitive\": \"float64\",\n",
      "                    \"form_key\": \"node10\"\n",
      "                }\n",
      "            },\n",
      "            \"form_key\": \"node6\"\n",
      "        },\n",
      "        \"pz\": {\n",
      "            \"class\": \"NumpyArray\",\n",
      "            \"itemsize\": 8,\n",
      "            \"format\": \"d\",\n",
      "            \"primitive\": \"float64\",\n",
      "            \"form_key\": \"node11\"\n",
      "        },\n",
      "        \"E\": {\n",
      "            \"class\": \"NumpyArray\",\n",
      "            \"itemsize\": 8,\n",
      "            \"format\": \"d\",\n",
      "            \"primitive\": \"float64\",\n",
      "            \"form_key\": \"node12\"\n",
      "        },\n",
      "        \"ex\": {\n",
      "            \"class\": \"ListOffsetArray64\",\n",
      "            \"offsets\": \"i64\",\n",
      "            \"content\": {\n",
      "                \"class\": \"ListOffsetArray64\",\n",
      "                \"offsets\": \"i64\",\n",
      "                \"content\": {\n",
      "                    \"class\": \"NumpyArray\",\n",
      "                    \"itemsize\": 1,\n",
      "                    \"format\": \"B\",\n",
      "                    \"primitive\": \"uint8\",\n",
      "                    \"parameters\": {\n",
      "                        \"__array__\": \"char\"\n",
      "                    },\n",
      "                    \"form_key\": \"node15\"\n",
      "                },\n",
      "                \"parameters\": {\n",
      "                    \"__array__\": \"string\"\n",
      "                },\n",
      "                \"form_key\": \"node14\"\n",
      "            },\n",
      "            \"form_key\": \"node13\"\n",
      "        }\n",
      "    },\n",
      "    \"form_key\": \"node0\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import awkward as ak\n",
    "array = ak.Array(\n",
    "\n",
    "[\n",
    "\n",
    "    {\"px\": [{\"key\": \"first key\", \"value\": 1}, {\"key\": \"second key\", \"value\": 2}], \"py\": {\"px\": True, \"py\": 3.2, \"pz\": 5.4, \"E\": 2.5}, \"pz\": 5.4, \"E\": 2.5, \"ex\": [\"fde\",\"fde\"]},\n",
    "\n",
    "    {\"px\": [{\"key\": \"first key\", \"value\": 1}, {\"key\": \"second key\", \"value\": 2}], \"py\": {\"px\": True, \"py\": 3.2, \"pz\": 5.4, \"E\": 2.5}, \"pz\": 543.34, \"E\": 24.12, \"ex\": [\"fde\",\"fde\"]},\n",
    "\n",
    "    {\"px\": [{\"key\": \"first key\", \"value\": 1}, {\"key\": \"second key\", \"value\": 2}], \"py\": {\"px\": True, \"py\": 3.2, \"pz\": 5.4, \"E\": 2.5}, \"pz\": 543.14, \"E\": 24.56, \"ex\": [\"fde\",\"fde\"]},\n",
    "\n",
    "],\n",
    "\n",
    ")\n",
    "a,b,c = ak.to_buffers(array)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c200adfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "valjson = json.loads(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1166106",
   "metadata": {},
   "outputs": [],
   "source": [
    "aa.field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57ab4f93",
   "metadata": {},
   "outputs": [],
   "source": [
    "import awkward as ak\n",
    "aa = ak.Array(ak.layout.ByteMaskedArray(\n",
    "    ak.layout.Index8([True,True,False,True,True,True]),\n",
    "    ak.layout.NumpyArray([1.3,4.5,13.56,5.64,45.6,2.3,]),valid_when = True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "187451d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b,c = ak.to_buffers(aa)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6017f8b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "ff = ak.Array(ak.layout.RegularArray(\n",
    "    ak.layout.NumpyArray([1.3,4.5,13.56,5.64,45.6,2.3,]), size = 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da49a69b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b,c = ak.to_buffers(ff)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5fdaf8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ak.layout.NumpyArray([b'hi',b'ho'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e3c4c6c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
